import requests
import re
import json
import threading

class qiancheng_wuyouSpider(threading.Thread):
    def  __init__(self, pageNumber, conn):
        threading.Thread.__init__(self)
        self.conn=conn
        self.pageNumber=pageNumber


    def run(self):
        # 连接数据库
        cur = self.conn.cursor()
        for page in range(1, self.pageNumber + 1):
            # 网址
            url = f'https://search.51job.com/list/000000,000000,0000,01%252c37%252c38,9,99,+,2,{page}.html'
            # 标头
            headers = {
                'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) '
                              'Chrome/95.0.4638.69 Safari/537.36 '
            }
            # 连接网页
            response = requests.get(url=url, headers=headers)
            # 数据在这个标签下
            html_data = re.findall('window.__SEARCH_RESULT__ = (.*?)</script>', response.text, re.S)[0]
            # json解析数据
            json_data = json.loads(html_data)

            for index in json_data['engine_jds']:
                # 数据
                try:
                    index['attribute_text'] = '|'.join(index['attribute_text'])
                    x = index['providesalary_text'][-3:-2]
                    y = index['providesalary_text'][-1]
                    salary = index['providesalary_text'][:-3].split('-')
                    if y == '年':
                        if x == '万':
                            maxSalary = float(salary[1]) * 10 / 12
                            minSalary = float(salary[0]) * 10 / 12
                        else:
                            maxSalary = float(salary[1])
                            minSalary = float(salary[0])
                    else:
                        if x == '万':
                            maxSalary = float(salary[1]) * 10
                            minSalary = float(salary[0]) * 10
                        else:
                            maxSalary = float(salary[1])
                            minSalary = float(salary[0])


                    d = index['attribute_text'].split('|')
                    # 数据库写入数据
                    sql = "INSERT INTO recruitment(Web,Job,Company,City,MinPay,MaxPay,Education,Experience," \
                          "CompanyType,Url) VALUES ('{:}','{:}','{:}', " \
                          "'{:}',{:.1f},{:.1f},'{:}','{:}','{:}','{:}') ".format('前程网', index['job_name'],
                                                                                 index['company_name'],
                                                                                 index['workarea_text'],
                                                                                  minSalary, maxSalary, d[2], d[1],
                                                                                 index['companytype_text'],
                                                                                 index['job_href'])
                except:
                    continue
                cur.execute(sql)
                self.conn.commit()
# qiancheng_wuyouSpider.spider(self=qiancheng_wuyouSpider,pageNumber=1,conn=database.operation.DateProcess.connect(database.operation.DateProcess))
